# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html

import random
import pymongo
from scrapy import signals, log
from scrapy.xlib.pydispatch import dispatcher
from scrapy.exceptions import IgnoreRequest
from scienet.user_agents import user_agent_list

class RandomUserAgentMiddleware(object):
    def process_request(self, request, spider):
        request.headers["User-Agent"] = random.choice(user_agent_list)
        if not request.headers.get("Host"):
            request.headers["Host"] = "blog.sciencenet.cn"

class HandleDataLossMiddleware(object):
    def process_response(self, request, response, spider):
        if "dataloss" in response.flags:
            log.msg("<%s>出现DataLoss！" % str(response.url), _level=log.WARNING)
            log.msg('返回状态码:' + str(response.status), _level=log.WARNING)
            request.dont_filter = True
            return request
        if "partial" in response.flags:
            log.msg("<%s>出现partial flag！" % str(response.url), _level=log.WARNING)
            log.msg('返回状态码:' + str(response.status), _level=log.WARNING)
            request.dont_filter = True
            return request
        return response

class HandleDuplicationMiddleware(object):
    def __init__(self):
        dispatcher.connect(self.open, signals.engine_started)
        dispatcher.connect(self.close, signals.engine_stopped)

    def open(self):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]
        self.collection = self.db[self.mongo_collection]

    def close(self):
        self.client.close()

    @classmethod
    def from_crawler(cls, crawler):
        cls.mongo_uri = crawler.settings.get("MONGO_URI", "localhost:27017")
        cls.mongo_db = crawler.settings.get("MONGO_DB", "scinet")
        cls.mongo_collection = crawler.settings.get("MONGO_COLLECTION", "bloggers")
        return cls()

    def process_request(self, request, spider):
        item = request.meta.get("item")
        if item:
            uid = item.get("uid")
            result = list(self.collection.find({"uid":uid}))
            if result:
                log.msg("博客id：" + uid + "->已存在数据库中！", _level=log.INFO)
                raise IgnoreRequest(request)
            else:
                log.msg("博客id：" + uid + "->不在数据库中，开始爬取！", _level=log.INFO)
